In [1]:
%pylab
%load_ext autoreload
%autoreload 2

from datetime import datetime, timedelta

import os
import sys

import numpy as np
import pandas as pd
from scipy import sparse
import sklearn as sl
import theanets as tn
from sklearn.cross_validation import StratifiedKFold, cross_val_score
from sklearn.feature_selection import RFECV
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.metrics import roc_auc_score, precision_recall_fscore_support

from cPickle import Pickler


Using matplotlib backend: Qt4Agg
Populating the interactive namespace from numpy and matplotlib
Couldn't import dot_parser, loading of dot files will not be possible.
Using gpu device 0: GeForce GTX 980

Load preprocessed train data


In [2]:
if os.name == 'nt':
    TRAIN_PATH = r'D:\train.csv'
    PTRAIN_PATH = r'D:\springleaf\train_preprocessed_all.csv'
    TEST_PATH = r'D:\test.csv'
    GOOGNEWS_PATH = r'D:\GoogleNews-vectors-negative300.bin.gz'
    VOCAB_PATH = r'D:\big.txt'
else:
    TRAIN_PATH = r'/media/mtambos/speedy/springleaf/train.csv'
    PTRAIN_PATH = r'/media/mtambos/speedy/springleaf/train_preprocessed_all.csv'
    TEST_PATH = r'/media/mtambos/speedy/springleaf/test.csv'
    GOOGNEWS_PATH = r'/media/mtambos/speedy/GoogleNews-vectors-negative300.bin.gz'
    VOCAB_PATH = r'/media/mtambos/speedy/big.txt'
df = pd.read_csv(PTRAIN_PATH, dtype=np.float32)

In [3]:
print "Calculating medians"
df_median = df.median(skipna=True)
print "Filling NAs"
df = df.fillna(value=df_median)


Calculating medians
Filling NAs

In [4]:
X = df.loc[:, df.columns != 'target']
y = df['target']

In [5]:
skf = StratifiedKFold(y, n_folds=5)

Random Forest Classifier


In [24]:
rf_classifier = RandomForestClassifier(n_estimators=100, n_jobs=-1, verbose=1, class_weight='auto')
rf_scores = cross_val_score(estimator=rf_classifier, X=X, y=y, scoring='roc_auc', cv=skf, verbose=2)
rf_scores


[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.0s remaining:  4.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   39.1s finished
[Parallel(n_jobs=8)]: Done   1 out of  47 | elapsed:    0.0s remaining:    1.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set -  40.0s
[Parallel(n_jobs=1)]: Done   1 jobs       | elapsed:   40.0s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.0s remaining:  4.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   38.1s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    2.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set -  39.1s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.9s remaining:  4.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   37.9s finished
[Parallel(n_jobs=8)]: Done   1 out of  37 | elapsed:    0.0s remaining:    0.8s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.3s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set -  38.8s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.0s remaining:  5.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   36.9s finished
[Parallel(n_jobs=8)]: Done   1 out of  52 | elapsed:    0.0s remaining:    1.0s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set -  37.8s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.9s remaining:  4.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   37.4s finished
[Parallel(n_jobs=8)]: Done   1 out of  85 | elapsed:    0.0s remaining:    2.0s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set -  38.3s
[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  3.2min finished
Out[24]:
array([ 0.75984939,  0.76191922,  0.75785135,  0.7627222 ,  0.75827303])

In [11]:
classifier.fit(X=X, y=y)
hundred_most_important_features = np.argpartition(classifier.feature_importances_, range(50))[:50]
df.columns[hundred_most_important_features]


[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    5.1s remaining:  8.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:  1.1min finished
Out[11]:
Index([u'VAR_0098', u'VAR_0106', u'VAR_0138', u'VAR_0191', u'VAR_0193',
       u'VAR_0214', u'VAR_0396', u'VAR_0445', u'VAR_0526', u'VAR_0529',
       u'VAR_0192', u'VAR_0107', u'VAR_0395', u'VAR_0397', u'VAR_0114',
       u'VAR_0194', u'VAR_0130', u'VAR_0398', u'VAR_0459', u'VAR_0399',
       u'VAR_0182', u'VAR_0463', u'VAR_0180', u'VAR_0131', u'VAR_0195',
       u'VAR_0392', u'VAR_0181', u'VAR_0411', u'VAR_0139', u'VAR_0393',
       u'VAR_0115', u'VAR_0449', u'VAR_0277', u'VAR_0108', u'VAR_0507',
       u'VAR_0099', u'VAR_0603', u'VAR_0468', u'VAR_0437', u'VAR_0275',
       u'VAR_0428', u'VAR_0386', u'VAR_0460', u'VAR_0371', u'VAR_1012',
       u'VAR_0156_year', u'VAR_1849', u'VAR_0476', u'VAR_0521', u'VAR_0229'],
      dtype='object')

In [14]:
hundred_most_important_features = sorted(classifier.feature_importances_, reverse=True)[:50]
hundred_most_important_features


Out[14]:
[0.0028886732507686784,
 0.0028240274642443606,
 0.0027145478135424438,
 0.002649123171116595,
 0.0026337071206189132,
 0.0025774113335795806,
 0.0025577297698921426,
 0.0024156090101467576,
 0.0023696137063085108,
 0.0023295319171059235,
 0.0022627918326271926,
 0.0022563778887519058,
 0.0022293222963358796,
 0.002217220855337247,
 0.002207549625493133,
 0.0021855396659237925,
 0.0021794922217861744,
 0.0021612128635470776,
 0.0021464625567381075,
 0.0020887284358406533,
 0.0020847885614810077,
 0.0020715077656085924,
 0.0020212881690338147,
 0.0019848917080766697,
 0.0019481709547825412,
 0.0019168199587121636,
 0.0019082153391379838,
 0.0018996953398093921,
 0.0018862264807112249,
 0.0018613543956342157,
 0.0018567161214247685,
 0.001840180358874736,
 0.0018355054539040527,
 0.0018309183724551155,
 0.0018186855540627753,
 0.001812398493039133,
 0.0018111956642958996,
 0.0018089679575626412,
 0.0018078433546637178,
 0.0018075085449324763,
 0.001804660508160783,
 0.0017879927547157614,
 0.0017817126624673359,
 0.0017520622281477821,
 0.0017424798374334751,
 0.0017367168562071355,
 0.0017351106598247736,
 0.0017306116877069367,
 0.0017259164970448659,
 0.0017151883570456045]

Extra Trees Classifier


In [22]:
et_classifier = ExtraTreesClassifier(n_estimators=100, n_jobs=-1, verbose=2, class_weight='auto')
et_scores = cross_val_score(estimator=et_classifier, X=X, y=y, scoring='roc_auc', cv=skf, verbose=2, n_jobs=-1)
et_scores


[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.4s remaining:  7.2min
[Parallel(n_jobs=-1)]: Done  51 out of 100 | elapsed:   30.9s remaining:   29.7s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   55.8s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    3.3s
[Parallel(n_jobs=8)]: Done  51 out of 100 | elapsed:    0.2s remaining:    0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 4 of 100
building tree 5 of 100
building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set -  56.8s
[Parallel(n_jobs=1)]: Done   1 jobs       | elapsed:   56.8s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    5.3s remaining:  8.7min
[Parallel(n_jobs=-1)]: Done  51 out of 100 | elapsed:   30.7s remaining:   29.5s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   56.0s finished
[Parallel(n_jobs=8)]: Done   1 out of  86 | elapsed:    0.0s remaining:    2.7s
[Parallel(n_jobs=8)]: Done  51 out of 100 | elapsed:    0.2s remaining:    0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 5 of 100
building tree 4 of 100
 building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set -  57.0s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.4s remaining:  7.2min
[Parallel(n_jobs=-1)]: Done  51 out of 100 | elapsed:   30.3s remaining:   29.1s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   54.6s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    3.3s
[Parallel(n_jobs=8)]: Done  51 out of 100 | elapsed:    0.3s remaining:    0.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 4 of 100
building tree 5 of 100
building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set -  55.6s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.3s remaining:  7.1min
[Parallel(n_jobs=-1)]: Done  51 out of 100 | elapsed:   29.3s remaining:   28.1s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   55.0s finished
[Parallel(n_jobs=8)]: Done   1 out of  57 | elapsed:    0.0s remaining:    1.9s
[Parallel(n_jobs=8)]: Done  51 out of 100 | elapsed:    0.3s remaining:    0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100
building tree 2 of 100
building tree 3 of 100
building tree 4 of 100
building tree 5 of 100
building tree 6 of 100
building tree 7 of 100
building tree 8 of 100
building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set -  56.1s
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.3s remaining:  7.1min
[Parallel(n_jobs=-1)]: Done  51 out of 100 | elapsed:   29.3s remaining:   28.2s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   53.7s finished
[Parallel(n_jobs=8)]: Done   1 out of  33 | elapsed:    0.0s remaining:    1.1s
[Parallel(n_jobs=8)]: Done  51 out of 100 | elapsed:    0.3s remaining:    0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
[CV] no parameters to be set .........................................
building tree 1 of 100building tree 2 of 100building tree 3 of 100building tree 4 of 100

 building tree 6 of 100building tree 7 of 100building tree 8 of 100
building tree 5 of 100




building tree 9 of 100
building tree 10 of 100
building tree 11 of 100
building tree 12 of 100
building tree 13 of 100
building tree 14 of 100
building tree 15 of 100
building tree 16 of 100
building tree 17 of 100
building tree 18 of 100
building tree 19 of 100
building tree 20 of 100
building tree 21 of 100
building tree 22 of 100
building tree 23 of 100
building tree 24 of 100
building tree 25 of 100
building tree 26 of 100
building tree 27 of 100
building tree 28 of 100
building tree 29 of 100
building tree 30 of 100
building tree 31 of 100
building tree 32 of 100
building tree 33 of 100
building tree 34 of 100
building tree 35 of 100
building tree 36 of 100
building tree 37 of 100
building tree 38 of 100
building tree 39 of 100
building tree 40 of 100
building tree 41 of 100
building tree 42 of 100
building tree 43 of 100
building tree 44 of 100
building tree 45 of 100
building tree 46 of 100
building tree 47 of 100
building tree 48 of 100
building tree 49 of 100
building tree 50 of 100
building tree 51 of 100
building tree 52 of 100
building tree 53 of 100
building tree 54 of 100
building tree 55 of 100
building tree 56 of 100
building tree 57 of 100
building tree 58 of 100
building tree 59 of 100
building tree 60 of 100
building tree 61 of 100
building tree 62 of 100
building tree 63 of 100
building tree 64 of 100
building tree 65 of 100
building tree 66 of 100
building tree 67 of 100
building tree 68 of 100
building tree 69 of 100
building tree 70 of 100
building tree 71 of 100
building tree 72 of 100
building tree 73 of 100
building tree 74 of 100
building tree 75 of 100
building tree 76 of 100
building tree 77 of 100
building tree 78 of 100
building tree 79 of 100
building tree 80 of 100
building tree 81 of 100
building tree 82 of 100
building tree 83 of 100
building tree 84 of 100
building tree 85 of 100
building tree 86 of 100
building tree 87 of 100
building tree 88 of 100
building tree 89 of 100
building tree 90 of 100
building tree 91 of 100
building tree 92 of 100
building tree 93 of 100
building tree 94 of 100
building tree 95 of 100
building tree 96 of 100
building tree 97 of 100
building tree 98 of 100
building tree 99 of 100
building tree 100 of 100
[CV] ................................ no parameters to be set -  54.7s
[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  4.7min finished
Out[22]:
array([ 0.75727822,  0.75785649,  0.75725815,  0.76027825,  0.75470654])

AdaBoost Classifier


In [28]:
ab_classifier = AdaBoostClassifier(n_estimators=100)
ab_scores = cross_val_score(estimator=ab_classifier, X=X, y=y, scoring='roc_auc', cv=skf, verbose=2, n_jobs=-1)
ab_scores


[Parallel(n_jobs=-1)]: Done   1 jobs       | elapsed: 13.3min
[Parallel(n_jobs=-1)]: Done   3 out of   5 | elapsed: 13.6min remaining:  9.1min
[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed: 13.6min finished
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] no parameters to be set .........................................
[CV] ................................ no parameters to be set -13.3min[CV] ................................ no parameters to be set -13.3min[CV] ................................ no parameters to be set -13.6min[CV] ................................ no parameters to be set -13.3min[CV] ................................ no parameters to be set -13.3min




Out[28]:
array([ 0.75965457,  0.76372955,  0.76249127,  0.76381007,  0.76312906])

Gradient Boost Classifier


In [ ]:
gb_classifier = GradientBoostingClassifier(n_estimators=100, verbose=1)
gb_scores = cross_val_score(estimator=gb_classifier, X=X, y=y, scoring='roc_auc', cv=skf,
                            verbose=2, n_jobs=4, pre_dispatch='n_jobs')
gb_scores

Recursive Feature Elimination


In [19]:
class RandomForestClassifierWithCoef(RandomForestClassifier):
    """
    Taken from http://stackoverflow.com/a/24656474/1632574
    """
    def fit(self, *args, **kwargs):
        super(RandomForestClassifierWithCoef, self).fit(*args, **kwargs)
        self.coef_ = self.feature_importances_

In [20]:
classifier2 = RandomForestClassifierWithCoef(n_estimators=100, n_jobs=-1, verbose=1)
rfecv = RFECV(estimator=classifier2, step=0.1, cv=skf, scoring='roc_auc', verbose=2)
rfecv.fit(X=X, y=y)
rfecv.n_features_


[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.5s remaining:  7.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   53.6s finished
Fitting estimator with 1905 features.
Fitting estimator with 1715 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.4s remaining:  7.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   51.4s finished
Fitting estimator with 1525 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.9s remaining:  6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   49.6s finished
Fitting estimator with 1335 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.8s remaining:  6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   47.2s finished
Fitting estimator with 1145 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.8s remaining:  6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   44.1s finished
Fitting estimator with 955 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.4s remaining:  5.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   41.6s finished
Fitting estimator with 765 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.2s remaining:  5.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   39.4s finished
Fitting estimator with 575 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.7s remaining:  4.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   34.9s finished
Fitting estimator with 385 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.5s remaining:  4.2min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   29.5s finished
Fitting estimator with 195 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    1.5s remaining:  2.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   19.5s finished
Fitting estimator with 5 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   21.5s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.5s finished
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   16.1s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.1s finished
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   16.9s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.0s finished
[Parallel(n_jobs=8)]: Done   1 out of   9 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.1s finished
Finished fold with 1 / 12 feature ranks, score=0.563906
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   19.5s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.5s finished
[Parallel(n_jobs=8)]: Done   1 out of  12 | elapsed:    0.0s remaining:    0.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.1s finished
Finished fold with 2 / 12 feature ranks, score=0.632698
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    1.6s remaining:  2.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   19.6s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    1.9s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.2s finished
Finished fold with 3 / 12 feature ranks, score=0.756228
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.3s remaining:  3.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   29.2s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    2.7s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.3s finished
Finished fold with 4 / 12 feature ranks, score=0.760178
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.8s remaining:  4.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   34.5s finished
[Parallel(n_jobs=8)]: Done   1 out of  98 | elapsed:    0.0s remaining:    1.7s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.3s finished
Finished fold with 5 / 12 feature ranks, score=0.760337
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.0s remaining:  5.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   38.4s finished
[Parallel(n_jobs=8)]: Done   1 out of  29 | elapsed:    0.0s remaining:    0.5s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 6 / 12 feature ranks, score=0.760583
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.1s remaining:  5.2min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   39.2s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    2.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.3s finished
Finished fold with 7 / 12 feature ranks, score=0.760868
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.5s remaining:  5.7min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   41.3s finished
[Parallel(n_jobs=8)]: Done   1 out of  69 | elapsed:    0.0s remaining:    1.6s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 8 / 12 feature ranks, score=0.758182
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.7s remaining:  6.1min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   43.9s finished
[Parallel(n_jobs=8)]: Done   1 out of  78 | elapsed:    0.0s remaining:    1.6s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 9 / 12 feature ranks, score=0.760035
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.6s remaining:  6.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   46.1s finished
[Parallel(n_jobs=8)]: Done   1 out of  23 | elapsed:    0.0s remaining:    0.5s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
Finished fold with 10 / 12 feature ranks, score=0.757785
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.1s remaining:  6.7min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   47.3s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    3.6s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
Finished fold with 11 / 12 feature ranks, score=0.759269
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.9s remaining:  6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   48.3s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    2.7s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 12 / 12 feature ranks, score=0.759254
Fitting estimator with 1905 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.9s remaining:  6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   47.0s finished
Fitting estimator with 1715 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.8s remaining:  6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   46.6s finished
Fitting estimator with 1525 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.0s remaining:  6.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   47.5s finished
Fitting estimator with 1335 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.0s remaining:  6.7min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   46.0s finished
Fitting estimator with 1145 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.8s remaining:  6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   43.8s finished
Fitting estimator with 955 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.8s remaining:  6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   43.8s finished
Fitting estimator with 765 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.1s remaining:  5.1min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   40.9s finished
Fitting estimator with 575 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.8s remaining:  4.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   35.0s finished
Fitting estimator with 385 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.4s remaining:  3.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   30.1s finished
Fitting estimator with 195 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    1.7s remaining:  2.8min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   20.7s finished
Fitting estimator with 5 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   17.4s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.2s finished
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   18.0s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.1s finished
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   15.2s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.0s finished
[Parallel(n_jobs=8)]: Done   1 out of  39 | elapsed:    0.0s remaining:    0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.1s finished
Finished fold with 1 / 12 feature ranks, score=0.562396
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    0.2s remaining:   16.8s
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    2.3s finished
[Parallel(n_jobs=8)]: Done   1 out of  15 | elapsed:    0.0s remaining:    0.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.2s finished
Finished fold with 2 / 12 feature ranks, score=0.639067
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    1.6s remaining:  2.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   20.2s finished
[Parallel(n_jobs=8)]: Done   1 out of  29 | elapsed:    0.0s remaining:    0.5s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.2s finished
Finished fold with 3 / 12 feature ranks, score=0.755461
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.4s remaining:  3.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   29.9s finished
[Parallel(n_jobs=8)]: Done   1 out of  18 | elapsed:    0.0s remaining:    0.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.3s finished
Finished fold with 4 / 12 feature ranks, score=0.758866
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    2.7s remaining:  4.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   37.7s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    2.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
Finished fold with 5 / 12 feature ranks, score=0.760468
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.4s remaining:  5.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   40.6s finished
[Parallel(n_jobs=8)]: Done   1 out of  95 | elapsed:    0.0s remaining:    1.8s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 6 / 12 feature ranks, score=0.760325
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.2s remaining:  5.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   43.9s finished
[Parallel(n_jobs=8)]: Done   1 out of  35 | elapsed:    0.0s remaining:    0.9s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 7 / 12 feature ranks, score=0.761522
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.3s remaining:  5.5min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   42.9s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    3.1s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
Finished fold with 8 / 12 feature ranks, score=0.761705
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.9s remaining:  6.4min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   45.4s finished
[Parallel(n_jobs=8)]: Done   1 out of 100 | elapsed:    0.0s remaining:    4.9s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 9 / 12 feature ranks, score=0.762252
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.8s remaining:  6.2min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   46.7s finished
[Parallel(n_jobs=8)]: Done   1 out of  47 | elapsed:    0.0s remaining:    1.3s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.5s finished
Finished fold with 10 / 12 feature ranks, score=0.761487
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.0s remaining:  6.6min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   47.8s finished
[Parallel(n_jobs=8)]: Done   1 out of   9 | elapsed:    0.0s remaining:    0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 11 / 12 feature ranks, score=0.762477
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.7s remaining:  6.0min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   49.9s finished
[Parallel(n_jobs=8)]: Done   1 out of   8 | elapsed:    0.0s remaining:    0.2s
[Parallel(n_jobs=8)]: Done 100 out of 100 | elapsed:    0.4s finished
Finished fold with 12 / 12 feature ranks, score=0.761509
Fitting estimator with 1905 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.2s remaining:  6.9min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   48.4s finished
Fitting estimator with 1715 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    3.8s remaining:  6.3min
[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   47.9s finished
Fitting estimator with 1525 features.
[Parallel(n_jobs=-1)]: Done   1 out of 100 | elapsed:    4.2s remaining:  7.0min
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-20-9fd5344f1c5e> in <module>()
      1 classifier2 = RandomForestClassifierWithCoef(n_estimators=100, n_jobs=-1, verbose=1)
      2 rfecv = RFECV(estimator=classifier2, step=0.1, cv=skf, scoring='roc_auc', verbose=2)
----> 3 rfecv.fit(X=X, y=y)
      4 rfecv.n_features_

/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/feature_selection/rfe.pyc in fit(self, X, y)
    374             # ranking_ contains the same set of values for all CV folds,
    375             # but perhaps reordered
--> 376             ranking_ = rfe.fit(X_train, y_train).ranking_
    377             # Score each subset of features
    378             for k in range(0, np.max(ranking_)):

/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/feature_selection/rfe.pyc in fit(self, X, y)
    159                 print("Fitting estimator with %d features." % np.sum(support_))
    160 
--> 161             estimator.fit(X[:, features], y)
    162 
    163             if estimator.coef_.ndim > 1:

<ipython-input-19-d94be2a1e7a5> in fit(self, *args, **kwargs)
      1 class RandomForestClassifierWithCoef(RandomForestClassifier):
      2     def fit(self, *args, **kwargs):
----> 3         super(RandomForestClassifierWithCoef, self).fit(*args, **kwargs)
      4         self.coef_ = self.feature_importances_

/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/ensemble/forest.pyc in fit(self, X, y, sample_weight)
    271                     t, self, X, y, sample_weight, i, len(trees),
    272                     verbose=self.verbose, class_weight=self.class_weight)
--> 273                 for i, t in enumerate(trees))
    274 
    275             # Collect newly grown trees

/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc in __call__(self, iterable)
    664                 # consumption.
    665                 self._iterating = False
--> 666             self.retrieve()
    667             # Make sure that we get a last message telling us we are done
    668             elapsed_time = time.time() - self._start_time

/home/mtambos/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc in retrieve(self)
    516                 self._lock.release()
    517             try:
--> 518                 self._output.append(job.get())
    519             except tuple(self.exceptions) as exception:
    520                 try:

/home/mtambos/anaconda/lib/python2.7/multiprocessing/pool.pyc in get(self, timeout)
    559 
    560     def get(self, timeout=None):
--> 561         self.wait(timeout)
    562         if not self._ready:
    563             raise TimeoutError

/home/mtambos/anaconda/lib/python2.7/multiprocessing/pool.pyc in wait(self, timeout)
    554         try:
    555             if not self._ready:
--> 556                 self._cond.wait(timeout)
    557         finally:
    558             self._cond.release()

/home/mtambos/anaconda/lib/python2.7/threading.pyc in wait(self, timeout)
    338         try:    # restore state no matter what (e.g., KeyboardInterrupt)
    339             if timeout is None:
--> 340                 waiter.acquire()
    341                 if __debug__:
    342                     self._note("%s.wait(): got it", self)

KeyboardInterrupt: